package com.cw.spider.parse.xinhua;

import cn.wanghaomiao.seimi.annotation.Crawler;
import cn.wanghaomiao.seimi.http.SeimiHttpType;
import cn.wanghaomiao.seimi.spring.common.CrawlerCache;
import cn.wanghaomiao.seimi.struct.Response;
import com.cw.spider.parse.AbstractCrawler;
import com.cw.spider.queue.Myqueue;
import com.cw.spider.service.NewService;

import org.seimicrawler.xpath.JXDocument;
import org.springframework.beans.factory.annotation.Autowired;

import java.util.List;
import java.util.stream.Collectors;

/**
 * 描述:
 *
 * @author withqianqian@163.com
 * @create 2020-08-27 9:08
 */
@Crawler(name = "xin_hua_basic", httpType = SeimiHttpType.OK_HTTP3, delay = 1,
        httpTimeOut = 3000, queue = Myqueue.class)
public class XinHuaCrawler extends AbstractCrawler {
    @Autowired
    XinHuaChildCrawler xinHuaChildCrawler;
    @Autowired
    XinHuaIndexCrawler xinHuaIndexCrawler;
    @Autowired
    NewService newService;

    @Override
    public String[] startUrls() {
        return new String[]{"http://www.xinhuanet.com"};
    }


    @Override
    public void start(Response response) {

        JXDocument doc = response.document();
        if (doc != null) {
            //导航
            List<Object> nUrls = doc.sel("//div[@id='navBody']//ul/li/a/@href");
            List<String> strNavUrls = nUrls
                    .stream()
                    .map(u -> {
                        if (u.toString().indexOf("http") > -1) {
                            return u.toString();
                        } else {
                            return "http://www.xinhuanet.com/" + u.toString();
                        }
                    })
                    .collect(Collectors.toList());

            xinHuaChildCrawler.setUrl(strNavUrls.toArray(new String[strNavUrls.size()]));


            //正文链接
            List<Object> pUrls = doc.sel("//div[@class='borderCont']/ul/li/a/@href");
            List<String> strPUrls = pUrls
                    .stream()
                    .filter(u -> u.toString().indexOf("http") > -1)
                    .map(u -> u.toString())
                    .collect(Collectors.toList());

            xinHuaIndexCrawler.setUrl(strPUrls.toArray(new String[strPUrls.size()]));
            CrawlerCache.getCrawlerModel("xin_hua_index_basic").startRequest();

        }

    }

}
